package scraper;

import java.io.FileNotFoundException;

import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.webharvest.definition.ScraperConfiguration;
import org.webharvest.runtime.Scraper;
import org.webharvest.runtime.variables.Variable;
//import mypackage.MyXmlLibrary;

public class testScrap {

    public static void main(String[] args) {
        try {
        	initLogger();
        	Logger LOGGER = Logger.getLogger(testScrap.class);
        	
			// register external plugins if there are any
			//DefinitionResolver.registerPlugin("com.my.MyPlugin3");

			ScraperConfiguration config = new ScraperConfiguration("config/amazonSearch2.xml");
			Scraper scraper = new Scraper(config, "/results/products/");
			//scraper.addVariableToContext("myXmlLib", new MyXmlLibrary());
			
			scraper.setDebug(false);
			scraper.execute();
			
			// takes variable created during execution
			//Variable url = (Variable) scraper.getContext().get("url");
			//LOGGER.info("=============================================");
			//LOGGER.info(url);
			
			Variable url2 = (Variable) scraper.getContext().get("url2");
			LOGGER.info("=============================================");
			LOGGER.info(url2);
			LOGGER.info("=============================================");
			

			
			for (Object obj : scraper.getContext().keySet()){
				LOGGER.info("OBJECT " + obj.toString());
			}
			// do something with articles...
		} catch (FileNotFoundException e) {
			System.out.println("File not found");
			e.printStackTrace();
		}
    }
	private static void initLogger(){
		PropertyConfigurator.configure("Z:\\WorkspaceGrails\\HTMLScraper\\config\\log4j.properties");
	}
} 